clear all
cd "sociology"
local list : dir . files "*.txt"
foreach f of local list {
di "`f'"
import delimited "`f'", encoding(ISO-8859-1)
split v1, generate(a)
quietly rename a1 b1
quietly egen newvar=concat(a*),punct(" ")
quietly rename b1 a1
quietly gen indexrecord=1 if a1=="RECORD"
quietly gen a3miss=1 if missing(a3)
quietly replace a3miss=0 if missing(a3miss)
quietly replace indexrecord=0 if (indexrecord==1 & a3miss==0)
quietly gen indexau=1 if a1=="AU"
quietly gen indexin=1 if a1=="IN"
quietly gen indexti=1 if a1=="TI"
quietly gen indexjn=1 if a1=="JN"
quietly gen indexap=1 if a1=="AP"
quietly gen indexyr=1 if a1=="YR"
quietly gen indexab=1 if a1=="AB"
quietly gen indexcdn=1 if a1=="CDN"
quietly gen indexis=1 if a1=="IS"
quietly gen indexmaj=1 if a1=="MAJ"
quietly gen indexid=1 if a1=="ID"
quietly gen indexcc=1 if a1=="CC"
quietly gen indexccd=1 if a1=="CCD"
quietly gen indexla=1 if a1=="LA"
quietly gen indexat=1 if a1=="AT"
quietly replace indexat=0 if (indexat==1 & a3miss==0)
quietly gen indexan=1 if a1=="AN"
quietly replace indexan=0 if (indexan==1 & a3miss==0)
quietly drop a3miss
quietly rename a1 b1
quietly drop a*
quietly rename b1 a1

foreach x of varlist index* {
quietly replace `x'=0 if missing(`x')
}
quietly egen sumindex = rowtotal(index*)
quietly replace newvar= a1+ " "+ newvar if sumindex==0
quietly replace a1="." if sumindex==0
quietly gen group=_n
quietly replace group= group[_n-1] if sumindex==0
quietly gen ord=_n
quietly sort group ord
quietly by group: gen order=_n

quietly by group: gen new=newvar
quietly by group: replace new = new[_n-1] + " " + new if _n >1
quietly by group: replace new=new[_N] if _n<_N

quietly replace new="." if sumindex==0
quietly gen groupauthor= group if indexau==1
quietly replace groupauthor=0 if (sumindex==1 & missing(groupauthor))
quietly replace groupauthor=-1 if missing(groupauthor)
quietly replace groupauthor=groupauthor[_n-1] if (groupauthor[_n-1]>0 & groupauthor==-1)
quietly replace new=newvar if groupauthor>0
quietly drop if new=="."&a1=="."

quietly gen articleid= new if indexrecord==1
quietly replace articleid=articleid[_n-1] if missing(articleid)
quietly gen author1=new if indexau==1
quietly gen author2=new if groupauthor>0 & order==2
quietly gen author3=new if groupauthor>0 & order==3
quietly gen author4=new if groupauthor>0 & order==4
quietly gen author5=new if groupauthor>0 & order==5
quietly gen author6=new if groupauthor>0 & order==6
quietly gen author7=new if groupauthor>0 & order==7
quietly gen year=new if indexyr==1
quietly gen journal=new if indexjn==1
quietly gen AP= new if indexap==1
quietly gen keywords= new if indexid==1
quietly gen language=new if indexla==1
quietly gen field=new if indexccd==1
quietly gen title=new if indexti==1

foreach x of varlist author1 author4 author5 author6 author7 year journal AP keywords language field title author2 author3{
quietly replace `x'=`x'[_n-1] if (articleid==articleid[_n-1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
quietly replace `x'=`x'[_n+1] if (articleid==articleid[_n+1]& missing(`x'))
}
quietly duplicates drop
quietly gen article="`f'"+" " + "articleid"
quietly keep article author1 author2 author3 author4 author5 author6 author7 year journal AP keywords language field title
save "sociology/`f'.dta", replace
set more off
clear
} 

clear all
cd "sociology"
local list : dir . files "*.dta"
foreach f of local list {
di "`f'"
append using "`f'"
set more off
}

gen articleid=_n
save "sociology.dta", replace

